﻿using System;
using System.Collections.Generic;
using System.Text;

namespace Workflows.Components.TextMining.Clustering
{
    /// <summary>
    /// max tfidf value across docs for a term is used
    /// </summary>
    public class TfIdfTermFilter : ITermFilter
    {
        #region fields
        private List<int> _ExcludedTermIDs;
        private double _TermFrequencyThreshold;
        #endregion

        #region ITermFilter Members

        public List<int> ExcludedTermIDs
        {
            get { return _ExcludedTermIDs; }
            set { _ExcludedTermIDs=value; }
        }

        public double TermFrequencyThreshold
        {
            get { return _TermFrequencyThreshold; }
            set { _TermFrequencyThreshold=value; }
        }

        public Dictionary<int, Dictionary<int, DocTermFreq>> FilterDocumentTerms(Dictionary<int, Dictionary<int, DocTermFreq>> allDocTermFreqs)
        {
            Dictionary<int, Dictionary<int, int>> tfs = new Dictionary<int, Dictionary<int, int>>(); // doc id -> (term id->term count)
            Dictionary<int, int> idfs = new Dictionary<int, int>();                                  // term id -> doc count
            foreach(int docID in allDocTermFreqs.Keys)
            {
                Dictionary<int, DocTermFreq> docTermFreqs = allDocTermFreqs[docID];
                Dictionary<int,int> tf=new Dictionary<int, int>();
                foreach (int termID in docTermFreqs.Keys)
                {
                    if (this._ExcludedTermIDs != null && this._ExcludedTermIDs.Contains(termID))
                        continue;

                    tf.Add(termID, docTermFreqs[termID].Count);
                    if(idfs.ContainsKey(termID))
                    {
                        idfs[termID] = idfs[termID] + 1;
                    }
                    else
                    {
                        idfs.Add(termID,1);
                    }
                }
                tfs.Add(docID,tf);
            }
            Dictionary<int,double> maxTfIdfScores=new Dictionary<int, double>();
            int docCount = allDocTermFreqs.Count;
            foreach(int docID in allDocTermFreqs.Keys)
            {
                Dictionary<int, DocTermFreq> docTermFreqs = allDocTermFreqs[docID];
                foreach(int termID in docTermFreqs.Keys)
                {
                    if(this._ExcludedTermIDs.Contains(termID))
                        continue;

                    double tfMeasure =(double) tfs[docID][termID];
                    double idfMeasure = idfs[termID];
                    double tfidfMeasure = tfMeasure*Math.Log10((double) docCount/idfMeasure);
                    if(maxTfIdfScores.ContainsKey(termID))
                    {
                        if(maxTfIdfScores[termID]<tfidfMeasure)
                        {
                            maxTfIdfScores[termID] = tfidfMeasure;
                        }
                    }
                    else
                    {
                        maxTfIdfScores.Add(termID,tfidfMeasure);
                    }
                }
            }
            Dictionary<int,Dictionary<int,DocTermFreq>> docTermFreqses2=new Dictionary<int, Dictionary<int, DocTermFreq>>();
            List<int> selectedTermIDs =new List<int>();
            foreach(int termID in maxTfIdfScores.Keys)
            {
                double score = maxTfIdfScores[termID];
                if(score>=this._TermFrequencyThreshold)
                {
                    selectedTermIDs.Add(termID);
                }
            }

            foreach(int docID in allDocTermFreqs.Keys)
            {
                Dictionary<int, DocTermFreq> docTermFreqs = allDocTermFreqs[docID];
                Dictionary<int,DocTermFreq> docTermFreqsFiltered=new Dictionary<int, DocTermFreq>();
                foreach(int termID in docTermFreqs.Keys)
                {
                    if(selectedTermIDs.Contains(termID))
                    {
                        docTermFreqsFiltered.Add(termID, docTermFreqs[termID]);
                    }
                }
                docTermFreqses2.Add(docID, docTermFreqsFiltered);
            }
            return docTermFreqses2;
        }

        #endregion
    }
}
